/*
 *  Announce
 *	Create-Time:2020-07-17
 *	Author: ChenXuanHong
 */

package spider

import (
	"ULCrawler/browser"
	//"ULCrawler/db"
	"ULCrawler/conf"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strconv"
	"strings"
	"time"

	"github.com/360EntSecGroup-Skylar/excelize"
	"github.com/tebeka/selenium"
)

const (
	DouYinLoginUrl       = "https://creator.douyin.com"
	DouYinViewUrl        = "https://creator.douyin.com/aweme/v1/creator/data/item/option"
	DouYinIdUrl          = "https://creator.douyin.com/aweme/v1/creator/data/item/list"
	DouYinCommentMainUrl = "https://creator.douyin.com/following/comment"
	DouYinCommentInfoUrl = "https://creator.douyin.com/aweme/v1/creator/comment/list/"

	DouYinDevLoginUrl     = "https://microapp.bytedance.com/"
	DouYinDevAppInforUrl  = "https://microapp.bytedance.com/api/v1/app/list"
	DouYinDevDataInforUrl = "https://microapp.bytedance.com/api/v1/app/%s/source_analysis"

	//新抖
	DouYinVideoUrl     = "https://xd.newrank.cn/?link=https%3A%2F%2Fxd.newrank.cn%2Fdata%2Fmaterial%2Fsearch"
	DouYinVideoIdUrl   = "https://xd.newrank.cn/xdnphb/nr/cloud/douyin/dou/getDouYinAccount"
	DouYinViedoDataUrl = "https://xd.newrank.cn/xdnphb/nr/cloud/douyin/dou/uploadVideo"

	DouYinAllUrl     = "https://e.douyin.com/site/manage-center/content-manage"
	DouYinAllIdUrl   = "https://e.douyin.com/aweme/v1/bluev/user/info"
	DouYinAllInfoUrl = "https://e.douyin.com/aweme/v1/saiyan/data/statistics"
)

type DouYinSpider struct {
	Client       *http.Client
	UserName     string
	PassWord     string
	UserName2    string
	PassWord2    string
	Company      string
	DouYinCookie string //抖音平台cookie (字节跳动)
	ChannelName  string
	done         map[string]bool
	appType      map[string]TouTiaoApp

	doFlag      chan bool
	token       string
	gameName    string
	posTypeMap  map[string]string
	videoIdMap  map[string]videoIdResult
	videoIdArr  []videoIdResult
	xinDouIdMap map[string]xinDouIdResult
	Sec_uid     string
	Wd          selenium.WebDriver
}
type videoIdResult struct {
	Link        string
	Id          string
	Create_time string
	Title       string
	Duration    int
	Sec_item_id string
}

type TouTiaoApp struct {
	AppId    string
	AppName  string
	TypeName string
}
type xinDouIdResult struct {
	NickName string
	Id       string
	OpenId   string
}

func NewDouYinSpider(userName, passWord, userName2, passWord2, Company, DouYinCookie string) (w *DouYinSpider, err error) {
	jar, err := cookiejar.New(nil)
	if err != nil {
		return
	}
	w = &DouYinSpider{
		UserName:     userName,
		PassWord:     passWord,
		UserName2:    userName2,
		PassWord2:    passWord2,
		Company:      Company,
		DouYinCookie: DouYinCookie,
		ChannelName:  "抖音",
		appType:      make(map[string]TouTiaoApp),
		Client: &http.Client{
			Jar: jar,
		},

		done:   make(map[string]bool),
		doFlag: make(chan bool, 1),
	}
	if err = w.login(); err != nil {
		return
	}

	time.Sleep(1 * time.Second)

	return
}
func (w *DouYinSpider) Post(b bool) {
	w.doFlag <- b
}

func (w *DouYinSpider) println(v ...interface{}) {
	str := fmt.Sprintf("[%s][%s][%s]", w.ChannelName, w.UserName, w.UserName2)
	log.Println(str, fmt.Sprintln(v...))
}

func (w *DouYinSpider) GetDone(date string) bool {
	return w.done[date]
}
func (w *DouYinSpider) SetDone(date string) {
	if !w.GetDone(date) {
		w.done = map[string]bool{date: true}
	}
}

func (w *DouYinSpider) login() error {
	err := w.login1()
	if err != nil {
		w.println("登录创作服务平台页失败,err=", err)
		return err
	}

	err = w.login3()
	if err != nil {
		w.println("登录新抖平台页失败,err=", err)
		return err
	}

	err = w.mockLogin2()
	if err != nil {
		w.println("登录开发者平台页失败,err=", err)
		return err
	}

	return err
}
func (w *DouYinSpider) login1() error {
	wd, err := browser.OpenWebDriver()
	if err != nil {
		w.println("创建浏览器对象失败，err= ", err)
		return err
	}
	//关闭一个webDriver会对应关闭一个chrome窗口
	//但是不会导致seleniumServer关闭
	//defer wd.Quit()
	err = wd.Get(DouYinLoginUrl)
	if err != nil {
		w.println("打开登陆网站失败,err= ", err)
		return err
	}
	time.Sleep(1 * time.Second)
	_, err = wd.ExecuteScript(`document.querySelector('.login').click();`, nil)
	if err != nil {
		w.println("点击登录失败,err= ", err)
		return err
	}
	time.Sleep(1 * time.Second)
	_, err = wd.ExecuteScript(`document.querySelector('.semi-button-content').click();`, nil)
	if err != nil {
		w.println("点击确认失败,err= ", err)
		return err
	}
	time.Sleep(1 * time.Second)
	w.println("请扫码登陆")
	// 判断是否登录成功
	time.Sleep(1 * time.Second)
	for {
		a, _ := wd.ExecuteScript(`return document.querySelectorAll('.avatar--1lU_a').length;`, nil)
		if a != nil && a.(float64) == 1 {
			break
		}
		time.Sleep(2 * time.Second)
	}
	time.Sleep(1 * time.Second)
	w.println("登陆成功")

	err = wd.Get(DouYinCommentMainUrl)
	if err != nil {
		w.println("login1跳转评论页失败 ", err)
		return err
	}

	w.Wd = wd
	time.Sleep(1 * time.Second)

	cookies, err := wd.GetCookies()
	if err != nil {
		w.println("获取cookies失败,err= ", err)
		return err
	}
	var clist []*http.Cookie
	for _, v := range cookies {
		clist = append(clist, &http.Cookie{
			Name:   v.Name,
			Value:  v.Value,
			Path:   v.Path,
			Domain: v.Domain,
		})
	}
	urlX, _ := url.Parse(DouYinLoginUrl)
	urlX2, _ := url.Parse(DouYinAllUrl)
	w.Client.Jar.SetCookies(urlX, clist)
	w.Client.Jar.SetCookies(urlX2, clist)

	time.Sleep(1 * time.Second)

	return nil
}

func (w *DouYinSpider) login2() error {
	wd, err := browser.OpenWebDriver()
	if err != nil {
		w.println("创建浏览器对象失败，err= ", err)
		return err
	}
	//关闭一个webDriver会对应关闭一个chrome窗口
	//但是不会导致seleniumServer关闭
	defer wd.Quit() // 关闭浏览器
	err = wd.Get(DouYinDevLoginUrl)
	if err != nil {
		w.println("打开抖音开发者平台网站失败,err= ", err)
		return err
	}
	time.Sleep(1 * time.Second)
	// 判断是否打开了网站
	tTimes := 0
	for {
		if t, _ := wd.Title(); t == "字节小程序开发者平台 - Creating future with excellent developers" {
			break
		}
		if tTimes > 20 {
			return errors.New("打开字节小程序开发者平台失败")
		}
		tTimes++
		time.Sleep(1 * time.Second)
	}
	if _, err = wd.ExecuteScript(`document.querySelector('.signup-btn').click();`, nil); err != nil {
		w.println("点击登陆弹出框失败")
		return err
	}
	time.Sleep(1 * time.Second)
	for {
		a, err := wd.ExecuteScript(`return document.querySelector('.ivu-modal').style.display;`, nil)
		if err == nil && a.(string) == "" {
			break
		}
		time.Sleep(1 * time.Second)
	}
	// 选择 手机登陆
	if _, err = wd.ExecuteScript(`document.querySelectorAll('.ivu-tabs-tab')[1].click();`, nil); err != nil {
		w.println("选择手机登陆失败")
		return err
	}
	time.Sleep(1 * time.Second)
	// 点击同意
	if _, err = wd.ExecuteScript(`document.querySelectorAll('.ivu-checkbox-input')[0].click();`, nil); err != nil {
		w.println("点击同意失败")
		return err
	}
	time.Sleep(1 * time.Second)
	es, err := wd.FindElements(selenium.ByCSSSelector, ".ivu-input-large")
	if err != nil {
		w.println("查找输入框失败")
		return err
	}
	// 输入账号
	if err = es[2].SendKeys(w.UserName); err != nil {
		w.println("输入账号失败")
		return err
	}
	time.Sleep(1 * time.Second)
	// 输入图片验证码
	/* reCode:
	var code string
	fmt.Print(fmt.Sprintf("正在登陆渠道:%s 账号:%s,请输入图片验证码:", w.ChannelName, w.UserName))
	fmt.Scanln(&code)
	// 输入图片验证码
	if err = es[4].Clear(); err != nil {
		w.println("清除图片输入框失败")
		return err
	}
	if err = es[4].SendKeys(code); err != nil {
		w.println("输入图片验证码失败")
		return err
	}
	time.Sleep(1 * time.Second)
	// 点击获取手机验证码
	if _, err = wd.ExecuteScript(`document.querySelectorAll('.mobile-code')[0].click();`, nil); err != nil {
		w.println("点击获取手机验证码失败")
		return err
	}
	// 判断图片验证码是否正确
	for {
		time.Sleep(1 * time.Second)
		a, _ := wd.ExecuteScript(`return document.querySelector('.ivu-message').children.length;`, nil)
		if a != nil && a.(float64) == 1 {
			goto reCode
		}
		b, _ := wd.ExecuteScript(`return document.querySelectorAll('.mobile-code')[0].disabled;`, nil)
		if b.(bool) {
			break
		}
	} */
	// 输入手机验证码
	// 点击获取手机验证码
	if _, err = wd.ExecuteScript(`document.querySelectorAll('.mobile-code')[0].click();`, nil); err != nil {
		w.println("点击获取手机验证码失败")
		return err
	}
	// 判断滑动验证码
	tips := false
	for {
		time.Sleep(1 * time.Second)
		if te, _ := wd.ExecuteScript(`return document.querySelector(".captcha_verify_bar--title div").innerText;`, nil); te == "请完成下列验证后继续:" && tips == false {
			w.println("请滑动图片验证码")
			tips = true
		}
		if _, err = wd.ExecuteScript(`return document.querySelector(".captcha_verify_bar--title div").innerText;`, nil); err != nil {
			break
		}
	}
reCode1:
	var code1 string
	fmt.Print(fmt.Sprintf("正在登陆渠道:%s 账号:%s,请输入手机验证码:", w.ChannelName, w.UserName))
	fmt.Scanln(&code1)
	if err = es[3].Clear(); err != nil {
		w.println("清除手机验证码输入框失败")
		return err
	}
	if err = es[3].SendKeys(code1); err != nil {
		w.println("输入手机验证码失败")
		return err
	}
	time.Sleep(1 * time.Second)
	// 点击登陆
	if _, err = wd.ExecuteScript(`document.querySelectorAll('.ivu-btn-long')[0].click();`, nil); err != nil {
		w.println("点击登陆失败")
		return err
	}
	// 判断是否登陆
	for {
		time.Sleep(1 * time.Second)
		a, _ := wd.ExecuteScript(`return document.querySelector('.ivu-message').children.length;`, nil)
		if a != nil && a.(float64) == 1 {
			goto reCode1
		}
		x, _ := wd.ExecuteScript(`return document.querySelectorAll('.ivu-modal-confirm-footer > button').length;`, nil)
		if x.(float64) == 1 {
			// 点击确定
			wd.ExecuteScript(`document.querySelector('.ivu-modal-confirm-footer > button').click();`, nil)
			time.Sleep(1 * time.Second)
		}
		n, _ := wd.ExecuteScript(`return document.querySelectorAll('.signup-btn').length;`, nil)
		if n.(float64) == 0 {
			break
		}
	}
	w.println("登陆成功")

	// 登陆成功
	time.Sleep(1 * time.Second)
	cookies, err := wd.GetCookies()
	if err != nil {
		w.println("获取cookies失败,err= ", err)
		return err
	}

	var clist []*http.Cookie
	var newCookie string
	for _, v := range cookies {
		//log.Println(v.Name,v.Value)
		clist = append(clist, &http.Cookie{
			Name:   v.Name,
			Value:  v.Value,
			Path:   v.Path,
			Domain: v.Domain,
		})
		newCookie = newCookie + fmt.Sprintf("%s=%s;", v.Name, v.Value)
	}
	urlX, _ := url.Parse(DouYinDevLoginUrl)
	w.Client.Jar.SetCookies(urlX, clist)

	err = w.WriteNewCookie(newCookie, "DouYinCookie")
	if err != nil {
		return err
	}

	return err
}

func (w *DouYinSpider) login3() error {
	wd, err := browser.OpenWebDriver()
	if err != nil {
		w.println("创建浏览器对象失败，err= ", err)
		return err
	}
	//关闭一个webDriver会对应关闭一个chrome窗口
	//但是不会导致seleniumServer关闭
	defer wd.Quit() // 关闭浏览器
	err = wd.Get(DouYinVideoUrl)
	if err != nil {
		w.println("打开新抖失败,err= ", err)
		return err
	}
	time.Sleep(1 * time.Second)

	// 判断是否打开了网站
	titleTimes := 0
	for {
		if titleTimes == 10 {
			w.println("新抖平台寻找title失败,平台title可能已经更换")
			return err
		}
		if t, _ := wd.Title(); t == "新抖-抖音视频及直播数据平台" || t == "新抖-抖音短视频直播带货数据分析平台" || t == "服务导航 - 新抖" {
			break
		}
		titleTimes++
		time.Sleep(1 * time.Second)
	}
	if _, err = wd.ExecuteScript(`document.querySelectorAll('.ant-btn')[2].click();`, nil); err != nil {
		w.println("点击登陆弹出框失败")
		return err
	}
	time.Sleep(1 * time.Second)
	loginBox := 0
	for {
		if loginBox == 10 {
			w.println("弹出框打开失败")
			return err
		}
		a, err := wd.ExecuteScript(`return document.querySelector('.nrd-login-modal-content').style.display;`, nil)
		if err == nil && a.(string) == "" {
			break
		}
		time.Sleep(1 * time.Second)
	}
	//选择 二维码 登录
	w.println("请扫描二维码登录")

	/* // 选择 手机登陆
	if _, err = wd.ExecuteScript(`document.querySelector('._2XRFN1F6').click();`, nil); err != nil {
		w.println("选择账号密码登陆失败")
		return err
	}
	time.Sleep(1 * time.Second)

	es, err := wd.FindElement(selenium.ByCSSSelector, "[placeholder='手机 / 邮箱 / 新榜ID']")
	if err != nil {
		w.println("查找账号输入框失败")
		return err
	}
	time.Sleep(1 * time.Second)

	// 输入账号
	if err = es.SendKeys(w.UserName2); err != nil {
		w.println("输入账号失败")
		return err
	}
	time.Sleep(1 * time.Second)

	es2, err := wd.FindElement(selenium.ByCSSSelector, "[placeholder='输入密码']")
	if err != nil {
		w.println("查找密码输入框失败")
		return err
	}
	time.Sleep(1 * time.Second)

	// 输入账号
	if err = es2.SendKeys(w.PassWord2); err != nil {
		w.println("输入密码失败")
		return err
	}
	time.Sleep(1 * time.Second)
	//点击登录
	_,err = wd.ExecuteScript(`document.querySelector('._38DPDVRd').click();`, nil)
	if err != nil {
		w.println("点击登录失败,err= ", err)
		return err
	}

	w.println("请滑动验证码") */
	// 判断是否登录成功
	for {
		t, err := wd.ExecuteScript(`return document.querySelectorAll('.ant-btn')[2].innerText;`, nil)
		if err != nil {
			w.println("登录失败，err=", err)
			return err
		}
		if t != "登录/注册" && err == nil {
			w.println("登录成功")
			break
		}
		time.Sleep(1 * time.Second)
	}

	// 登陆成功
	time.Sleep(1 * time.Second)
	cookies, err := wd.GetCookies()
	if err != nil {
		w.println("获取cookies失败,err= ", err)
		return err
	}

	var clist []*http.Cookie

	for _, v := range cookies {
		//log.Println(v.Name,v.Value)
		clist = append(clist, &http.Cookie{
			Name:   v.Name,
			Value:  v.Value,
			Path:   v.Path,
			Domain: v.Domain,
		})

	}
	urlX, _ := url.Parse(DouYinVideoUrl)
	w.Client.Jar.SetCookies(urlX, clist)

	return err
}
func (w *DouYinSpider) mockLogin2() (err error) {
	var clist []*http.Cookie
	str := w.DouYinCookie
	if str == "" {
		return errors.New(w.UserName + "不存在douYinCookie字段")
	}
	arr1 := strings.Split(str, ";")
	for _, v := range arr1 {
		arr2 := strings.Split(v, "=")
		if len(arr2) > 1 {
			clist = append(clist, &http.Cookie{
				Name:  arr2[0],
				Value: arr2[1],
			})
		}
	}
	urlX, _ := url.Parse(DouYinDevLoginUrl)
	w.Client.Jar.SetCookies(urlX, clist)
	return
}
func (w *DouYinSpider) WriteNewCookie(newCookie, cookieType string) (err error) {
	var AcountArr = conf.Conf.DouYinAcount
	flag := false
	for i, v := range AcountArr {
		if v.DouYinCookie == w.DouYinCookie {
			if cookieType == "DouYinCookie" {
				v.DouYinCookie = newCookie
				AcountArr[i] = v
			} else {
				return errors.New("写入cookie失败,key=" + cookieType)
			}
			flag = true
		}
	}
	conf.Conf.DouYinAcount = AcountArr
	jsonStr, err := json.Marshal(conf.Conf)
	if err != nil {
		w.println("WriteNewCookie解析失败")
		return
	}
	//将新Cookie重新写入配置文件
	err = ioutil.WriteFile("conf.json", jsonStr, 777)
	if err != nil {
		panic(err)
	}
	if flag == false {
		return errors.New("没有找到对应账号的cookie")
	}
	return
}

type DouYinIdResult struct {
	Status_code    int `json:"status_code"`
	Item_info_list []struct {
		Item_id     string `json:"item_id"`     //视频ID
		Item_link   string `json:"item_link"`   //视频链接
		Create_time string `json:"create_time"` //发布日期
		Title       string `json:"title"`       //标题
		Duration    int    `json:"duration"`    //时长
		Sec_item_id string `json:"sec_item_id"` //评论管理需要的item_id
	} `json:"item_info_list"`
}

//登录后获得videoId列表
func (w *DouYinSpider) getVideoId() (err error) {
re:
	w.videoIdMap = make(map[string]videoIdResult)
	w.videoIdArr = make([]videoIdResult, 0)
	data := url.Values{}
	data.Add("count", "1000")
	data.Add("cursor", "0")
	req, err := http.NewRequest("GET", DouYinIdUrl+"?"+data.Encode(), nil)
	if err != nil {
		return err
	}
	req.Header.Set("referer", "https://creator.douyin.com/data/stats/video")
	resp, err := w.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var p DouYinIdResult
	err = json.Unmarshal(b, &p)
	if err != nil {
		w.println("(getVideoId)JSON解析失败", string(b))
		return err
	}
	if err != nil || p.Status_code != 0 {
		w.println("(getVideoId)登陆失效，重新登陆中。。。err=", err, string(b))
	login:
		if err = w.login1(); err == nil {
			w.println("登录成功")
		} else {
			w.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
			goto login
		}
		goto re
	}
	for _, v := range p.Item_info_list {
		w.videoIdMap[v.Item_id] = videoIdResult{
			Link:        v.Item_link,
			Id:          v.Item_id,
			Create_time: v.Create_time,
			Title:       v.Title,
			Duration:    v.Duration,
			Sec_item_id: v.Sec_item_id,
		}
		s := videoIdResult{
			Link:        v.Item_link,
			Id:          v.Item_id,
			Create_time: v.Create_time,
			Title:       v.Title,
			Duration:    v.Duration,
			Sec_item_id: v.Sec_item_id,
		}
		w.videoIdArr = append(w.videoIdArr, s)
	}
	return
}

type DouYinViewResult struct {
	Status_code int `json:"status_code"`
	Option_list []struct {
		Date  string `json:"date"`
		Count string `json:"count"`
	} `json:"option_list"`
}
type DouYinView struct {
	Title              string
	Duration           string
	Link               string
	Date               string
	VideoId            string
	ViewCount          string
	LikeCount          string
	CommentCount       string
	ShareCount         string
	LikeRate           string
	CommentRate        string
	ShareRate          string
	InteractionRate    string
	PlayCompletionRate string
	AveragePlayTime    string
	VideoAllTime       string
	VideoWithFans      string
}

func (w *DouYinSpider) collectView(date string) (cs [][]DouYinView, err error) {
re:
	cs = make([][]DouYinView, 0)
	data := url.Values{}
	data2 := url.Values{}
	data3 := url.Values{}
	data4 := url.Values{}
	data5 := url.Values{}
	data6 := url.Values{}
	data7 := url.Values{}
	//播放
	data.Add("option_type", "1")
	data.Add("last_days_type", "3")
	//点赞
	data2.Add("option_type", "3")
	data2.Add("last_days_type", "3")
	//评论
	data3.Add("option_type", "4")
	data3.Add("last_days_type", "3")
	//分享
	data4.Add("option_type", "5")
	data4.Add("last_days_type", "3")
	//播放完成率
	data5.Add("option_type", "7")
	data5.Add("last_days_type", "3")
	//平均播放时长
	data6.Add("option_type", "8")
	data6.Add("last_days_type", "3")
	//视频带粉
	data7.Add("option_type", "25")
	data7.Add("last_days_type", "3")
	for _, s := range w.videoIdArr {
		cq := []DouYinView{}
		data.Set("item_id", s.Id)
		data2.Set("item_id", s.Id)
		data3.Set("item_id", s.Id)
		data4.Set("item_id", s.Id)
		data5.Set("item_id", s.Id)
		data6.Set("item_id", s.Id)
		data7.Set("item_id", s.Id)
		req, err := http.NewRequest("GET", DouYinViewUrl+"?"+data.Encode(), nil)
		if err != nil {
			return nil, err
		}
		req.Header.Set("referer", "https://creator.douyin.com/data/stats/video")

		req2, err := http.NewRequest("GET", DouYinViewUrl+"?"+data2.Encode(), nil)
		if err != nil {
			return nil, err
		}
		req2.Header.Set("referer", "https://creator.douyin.com/data/stats/video")

		req3, err := http.NewRequest("GET", DouYinViewUrl+"?"+data3.Encode(), nil)
		if err != nil {
			return nil, err
		}
		req3.Header.Set("referer", "https://creator.douyin.com/data/stats/video")

		req4, err := http.NewRequest("GET", DouYinViewUrl+"?"+data4.Encode(), nil)
		if err != nil {
			return nil, err
		}
		req4.Header.Set("referer", "https://creator.douyin.com/data/stats/video")

		req5, err := http.NewRequest("GET", DouYinViewUrl+"?"+data5.Encode(), nil)
		if err != nil {
			return nil, err
		}
		req5.Header.Set("referer", "https://creator.douyin.com/data/stats/video")

		req6, err := http.NewRequest("GET", DouYinViewUrl+"?"+data6.Encode(), nil)
		if err != nil {
			return nil, err
		}
		req6.Header.Set("referer", "https://creator.douyin.com/data/stats/video")

		req7, err := http.NewRequest("GET", DouYinViewUrl+"?"+data7.Encode(), nil)
		if err != nil {
			return nil, err
		}
		req7.Header.Set("referer", "https://creator.douyin.com/data/stats/video")

	reResp:
		resp, err := w.Client.Do(req)
		if err != nil {
			w.println("collectView1")
			time.Sleep(1 * time.Second)
			goto reResp
		}
		time.Sleep(200 * time.Millisecond)

	reResp2:
		resp2, err := w.Client.Do(req2)
		if err != nil {
			w.println("collectView2")
			time.Sleep(1 * time.Second)
			goto reResp2
		}
		time.Sleep(200 * time.Millisecond)

	reResp3:
		resp3, err := w.Client.Do(req3)
		if err != nil {
			w.println("collectView3")
			time.Sleep(1 * time.Second)
			goto reResp3
		}
		time.Sleep(200 * time.Millisecond)

	reResp4:
		resp4, err := w.Client.Do(req4)
		if err != nil {
			w.println("collectView4")
			time.Sleep(1 * time.Second)
			goto reResp4
		}
		time.Sleep(200 * time.Millisecond)

	reResp5:
		resp5, err := w.Client.Do(req5)
		if err != nil {
			w.println("collectView5")
			time.Sleep(1 * time.Second)
			goto reResp5
		}
		time.Sleep(200 * time.Millisecond)

	reResp6:
		resp6, err := w.Client.Do(req6)
		if err != nil {
			w.println("collectView6")
			time.Sleep(1 * time.Second)
			goto reResp6
		}
		time.Sleep(200 * time.Millisecond)

	reResp7:
		resp7, err := w.Client.Do(req7)
		if err != nil {
			w.println("collectView7")
			time.Sleep(1 * time.Second)
			goto reResp7
		}
		defer resp.Body.Close()
		defer resp2.Body.Close()
		defer resp3.Body.Close()
		defer resp4.Body.Close()
		defer resp5.Body.Close()
		defer resp6.Body.Close()
		defer resp7.Body.Close()
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		b2, err := ioutil.ReadAll(resp2.Body)
		if err != nil {
			return nil, err
		}
		b3, err := ioutil.ReadAll(resp3.Body)
		if err != nil {
			return nil, err
		}
		b4, err := ioutil.ReadAll(resp4.Body)
		if err != nil {
			return nil, err
		}
		b5, err := ioutil.ReadAll(resp5.Body)
		if err != nil {
			return nil, err
		}
		b6, err := ioutil.ReadAll(resp6.Body)
		if err != nil {
			return nil, err
		}
		b7, err := ioutil.ReadAll(resp7.Body)
		if err != nil {
			return nil, err
		}
		var p, p2, p3, p4, p5, p6, p7 DouYinViewResult
		err = json.Unmarshal(b, &p)
		if err != nil {
			w.println("(collectView)JSON解析失败", string(b))
			return nil, err
		}
		err = json.Unmarshal(b2, &p2)
		if err != nil {
			w.println("(collectView2)JSON解析失败", string(b))
			return nil, err
		}
		err = json.Unmarshal(b3, &p3)
		if err != nil {
			w.println("(collectView3)JSON解析失败", string(b))
			return nil, err
		}
		err = json.Unmarshal(b4, &p4)
		if err != nil {
			w.println("(collectView4)JSON解析失败", string(b))
			return nil, err
		}
		err = json.Unmarshal(b5, &p5)
		if err != nil {
			w.println("(collectView5)JSON解析失败", string(b))
			return nil, err
		}
		err = json.Unmarshal(b6, &p6)
		if err != nil {
			w.println("(collectView6)JSON解析失败", string(b))
			return nil, err
		}
		err = json.Unmarshal(b7, &p7)
		if err != nil {
			w.println("(collectView7)JSON解析失败", string(b))
			return nil, err
		}
		//登录失效返回是非JSON，JSON解析失败
		if p.Status_code != 0 || p2.Status_code != 0 || p3.Status_code != 0 || p4.Status_code != 0 || p5.Status_code != 0 || p6.Status_code != 0 || p7.Status_code != 0 {
			w.println("(collectView)登陆失效，重新登陆中。。。", string(b), s.Id, p.Status_code, p2.Status_code, p3.Status_code, p4.Status_code, p5.Status_code, p6.Status_code, p7.Status_code)
		login:
			if err = w.login1(); err == nil {
				w.println("登录成功")
			} else {
				w.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
				goto login
			}
			goto re
		}
		for _, v := range p.Option_list {
			c := DouYinView{
				Title:     s.Title,
				Duration:  fmt.Sprint(s.Duration),
				Link:      s.Link,
				Date:      v.Date,
				ViewCount: v.Count,
				VideoId:   s.Id,
			}
			cq = append(cq, c)
		}
		for ind, val := range cq {
			for _, v2 := range p2.Option_list {
				if val.Date == v2.Date && val.VideoId == s.Id {
					cq[ind].LikeCount = v2.Count
				}
			}
			for _, v3 := range p3.Option_list {
				if val.Date == v3.Date && val.VideoId == s.Id {
					cq[ind].CommentCount = v3.Count
				}
			}
			for _, v4 := range p4.Option_list {
				if val.Date == v4.Date && val.VideoId == s.Id {
					cq[ind].ShareCount = v4.Count
				}
			}
			for _, v5 := range p5.Option_list {
				if val.Date == v5.Date && val.VideoId == s.Id {
					k, _ := strconv.ParseFloat(v5.Count, 64)
					cq[ind].PlayCompletionRate = fmt.Sprintf("%.2f", k*100) + "%"
				}
			}
			for _, v6 := range p6.Option_list {
				if val.Date == v6.Date && val.VideoId == s.Id {
					cq[ind].AveragePlayTime = v6.Count
				}
			}
			for _, v7 := range p7.Option_list {
				if val.Date == v7.Date && val.VideoId == s.Id {
					cq[ind].VideoWithFans = v7.Count
				}
			}
			if val.VideoId == s.Id {
				View, _ := strconv.ParseFloat(cq[ind].ViewCount, 64)
				Like, _ := strconv.ParseFloat(cq[ind].LikeCount, 64)
				Comment, _ := strconv.ParseFloat(cq[ind].CommentCount, 64)
				Share, _ := strconv.ParseFloat(cq[ind].ShareCount, 64)
				if View > 0 {
					cq[ind].LikeRate = fmt.Sprintf("%.2f", (Like/View)*100) + "%"
					cq[ind].CommentRate = fmt.Sprintf("%.2f", (Comment/View)*100) + "%"
					cq[ind].ShareRate = fmt.Sprintf("%.2f", (Share/View)*100) + "%"
					cq[ind].InteractionRate = fmt.Sprintf("%.2f", ((Like+Comment+Share)/View)*100) + "%"
					cq[ind].VideoAllTime = fmt.Sprint(int(float64((s.Duration / 1000)) + 0.5))
				} else {
					cq[ind].LikeRate = "0%"
					cq[ind].CommentRate = "0%"
					cq[ind].ShareRate = "0%"
					cq[ind].InteractionRate = "0%"
					cq[ind].VideoAllTime = "0%"
				}
			}
		}
		cs = append(cs, cq)
	}

	return
}

// 获取app信息
type AppInfoResult struct {
	Error int `json:"error"`
	Data  struct {
		CreatorList      []TouTiaoAppInfo `json:"creator_list"`
		AdminList        []TouTiaoAppInfo `json:"admin_list"`
		CollaboratorList []TouTiaoAppInfo `json:"collaborator_list"`
	} `json:"data"`
}
type TouTiaoAppInfo struct {
	Appid string `json:"appid"`
	Type  int    `json:"type"`
	Name  string `json:"name"`
	State int    `json:"state"`
}

func (t *DouYinSpider) getAppInfo() error {
re:
	t.appType = make(map[string]TouTiaoApp)
	req, err := http.NewRequest("GET", DouYinDevAppInforUrl, nil)
	if err != nil {
		return err
	}
	resp, err := t.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var a AppInfoResult
	err = json.Unmarshal(b, &a)
	if err != nil {
		t.println("(getAppInfo)JSON解析失败", string(b))
		return err
	}
	if a.Error != 0 {
		t.println("(getAppInfo)登陆失效，重新登陆中。。。", string(b))
	login:
		if err = t.login2(); err == nil {
			t.println("登陆成功")
		} else {
			t.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
			goto login
		}
		goto re
	}

	for _, v := range a.Data.CreatorList {
		if v.State == 1 {
			var tn string
			if v.Type == 2 {
				tn = "小游戏"
			} else {
				tn = "小游戏"
			}
			t.appType[v.Appid] = TouTiaoApp{
				AppId:    v.Appid,
				AppName:  v.Name,
				TypeName: tn,
			}
		}
	}
	for _, v := range a.Data.AdminList {
		if v.State == 1 {
			var tn string
			if v.Type == 2 {
				tn = "小游戏"
			} else {
				tn = "小游戏"
			}
			t.appType[v.Appid] = TouTiaoApp{
				AppId:    v.Appid,
				AppName:  v.Name,
				TypeName: tn,
			}
		}
	}
	for _, v := range a.Data.CollaboratorList {
		if v.State == 1 {
			var tn string
			if v.Type == 2 {
				tn = "小游戏"
			} else {
				tn = "小游戏"
			}
			t.appType[v.Appid] = TouTiaoApp{
				AppId:    v.Appid,
				AppName:  v.Name,
				TypeName: tn,
			}
		}
	}
	return nil
}

//开发者平台
type DouYinDevDataInfo struct {
	Message string `json:"message"`
	Data    struct {
		Series []struct {
			Suffix_key string `json:"suffix_key"`
			Data       []int  `json:"data"`
		} `json:"series"`
	} `json:"data"`
}
type DouYinDevData struct {
	Mount   string
	Date    string
	AppName string
	AppId   string
}

func (w *DouYinSpider) collectDevDataInfo(date string) (as [][]DouYinDevData, err error) {
	reTimes := 0
re:
	as = make([][]DouYinDevData, 0)
	datetime := date + " 00:00:00"
	timeLayout := "2006-01-02 15:04:05"
	loc, _ := time.LoadLocation("Local")
	theTime, _ := time.ParseInLocation(timeLayout, datetime, loc)
	timestamp := theTime.Unix() - 2592000
	for _, s := range w.appType {
		aq := make([]DouYinDevData, 0)
		data := `{"period":{"interval":"day","range":[%d,%d],"type":"range"},"hostId":1128,"queryEvents":[{"label":"新增用户数","eventName":"mp_launch","resultType":{"type":"event_users"},"groupBy":["scene"],"attrs":[{"attr":"_app_name","content_type":"event_param","op":"=","type":"string","value":["aweme"]}]}],"filters":[{"name":"新用户","attr":"user_is_new","op":"=","value":[1]}]}`
		reader := strings.NewReader(fmt.Sprintf(data, timestamp, timestamp+2592000))
		req, err := http.NewRequest("POST", fmt.Sprintf(DouYinDevDataInforUrl, s.AppId), reader)
		if err != nil {
			return nil, err
		}
		//req.Header.Set("Content-Type", "application/json")

		resp, err := w.Client.Do(req)
		if err != nil {
			return nil, err
		}

		defer resp.Body.Close()

		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}

		var r DouYinDevDataInfo

		err = json.Unmarshal(b, &r)
		if err != nil {
			if reTimes < 6 {
				w.println("(collectDevDataInfo)JSON解析失败,尝试重新请求,times:"+fmt.Sprint(reTimes), string(b))
				goto re
			}
			w.println("(collectDevDataInfo)JSON解析失败,结束"+fmt.Sprint(reTimes), string(b))
			return nil, err
		}
		if r.Message != "success" {
			w.println("(collectDevDataInfo)登陆失效，重新登陆中。。。", string(b), err)
		login:
			if err = w.login2(); err == nil {
				w.println("登陆成功")
			} else {
				w.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
				goto login
			}
			goto re
		}
		timestamp2 := timestamp
		for _, v := range r.Data.Series {
			if v.Suffix_key == "023001" {
				for _, vv := range v.Data {
					timeLayout := "2006-01-02"
					datetime := time.Unix(int64(timestamp2), 0).Format(timeLayout)
					a := DouYinDevData{
						Mount:   fmt.Sprint(vv),
						Date:    datetime,
						AppName: s.AppName,
						AppId:   s.AppId,
					}
					aq = append(aq, a)
					timestamp2 = timestamp2 + 86400

				}
			}
		}
		if len(aq) > 0 {
			as = append(as, aq)
		}
	}

	return
}

//获取新抖平台openId
type XinDouId struct {
	Code int `json:"code"`
	Data struct {
		Owner []struct {
			Open_id  string `json:"open_id"`
			Id       string `json:"id"`
			NickName string `json:"nickname"`
		} `json:"owner"`
	} `json:"data"`
}

func (w *DouYinSpider) getOpenId() error {
	w.xinDouIdMap = make(map[string]xinDouIdResult)
re:
	req, err := http.NewRequest("POST", DouYinVideoIdUrl, nil)
	if err != nil {
		return err
	}
	resp, err := w.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	var a XinDouId
	err = json.Unmarshal(b, &a)
	if err != nil {
		w.println("(getOpenId)JSON解析失败", string(b))
		return err
	}
	if a.Code != 2000 {
		w.println("(getOpenId)登陆失效，重新登陆中。。。", string(b))
	login:
		if err = w.login3(); err == nil {
			w.println("登陆成功")
		} else {
			w.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
			goto login
		}
		goto re
	}
	for _, v := range a.Data.Owner {
		w.xinDouIdMap[v.Id] = xinDouIdResult{
			OpenId:   v.Open_id,
			Id:       v.Id,
			NickName: v.NickName,
		}
	}
	return nil
}

//新抖平台数据
type XinDouDataInfo struct {
	Code int `json:"code"`
	Data struct {
		Cursor   interface{} `json:"cursor"`
		Get_time string      `json:"get_time"`
		List     []struct {
			Title          string `json:"title"`          //标题
			Create_time    string `json:"create_time"`    //发布时间
			Comment_count  string `json:"comment_count"`  //评论数
			Forward_count  string `json:"forward_count"`  //转发数
			Play_count     string `json:"play_count"`     //播放数
			Download_count string `json:"download_count"` //下载数
			Share_count    string `json:"share_count"`    //分享数
			Digg_count     string `json:"digg_count"`     //获赞数
			Share_url      string `json:"share_url"`      //播放链接
			Is_reviewed    string `json:"is_reviewed"`    //是否已审核
			Item_id        string `json:"item_id"`        //ID
		} `json:"list"`
	} `json:"data"`
}
type XinDouData struct {
	Title          string
	Create_time    string
	Is_reviewed    string
	Share_url      string
	Play_count     string
	Digg_count     string
	Comment_count  string
	Forward_count  string
	Share_count    string
	Download_count string
	Item_id        string
	Digg_rate      string
	Share_rate     string
	Comment_rate   string
}

func (w *DouYinSpider) collectXinDouDataInfo(date string) (ps []XinDouData, err error) {
	ps = make([]XinDouData, 0)
	var CompanyId xinDouIdResult
	for _, v := range w.xinDouIdMap {
		if strings.Contains(v.NickName, w.Company) {
			CompanyId = v
			break
		}
	}
	if CompanyId.OpenId == "" {
		w.println("未授权.......")
		return nil, nil
	}

	data := `{"openId":"%s","id":"%s","nickname":"%s","get_time":"%s"}`
	reader := strings.NewReader(fmt.Sprintf(data, CompanyId.OpenId, CompanyId.Id, CompanyId.NickName, date))
	req, err := http.NewRequest("POST", DouYinViedoDataUrl, reader)
	if err != nil {
		return nil, err
	}
	req.Header.Set("content-Type", "application/json;charset=UTF-8")

	resp, err := w.Client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	r := bytes.NewReader(b)
	w.XinDouExcel(r)

	/* var r XinDouDataInfo
	err = json.Unmarshal(b, &r)

	if err != nil {
		w.println("(collectXinDouDataInfo)JSON解析失败", string(b))
		w.println("params",fmt.Sprintf(data,CompanyId,cursor))
		return nil ,err
	}
	if r.Code != 2000{
		w.println("(collectXinDouDataInfo)登陆失效，重新登陆中。。。", string(b),err)
		if strings.Contains(string(b),`"code":5000`){
			w.println("你的新抖平台授权可能已经过期,请先检查")
		}
	login:
		if err = w.login3(); err == nil {
			w.println("登陆成功")
		} else {
			w.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
			goto login
		}
		goto re
	}
	if len(r.Data.List) != 0{
		for _,v := range r.Data.List{
			Play,_ := strconv.ParseFloat(v.Play_count,64)
			Digg,_ := strconv.ParseFloat(v.Digg_count,64)
			Share,_ := strconv.ParseFloat(v.Share_count,64)
			Comment,_ := strconv.ParseFloat(v.Comment_count,64)
			if v.Is_reviewed == "true"{
				v.Is_reviewed = "是"
			}else if v.Is_reviewed == "false"{
				v.Is_reviewed = "否"
			}
			p := XinDouData{
				Item_id   		:	v.Item_id,
				Title  			:	v.Title,
				Create_time   	:	v.Create_time,
				Is_reviewed   	:	v.Is_reviewed,
				Share_url   	:	v.Share_url,
				Play_count   	:	v.Play_count,
				Digg_count   	:	v.Digg_count,
				Comment_count   :	v.Comment_count,
				Forward_count   :	v.Forward_count,
				Share_count   	:	v.Share_count,
				Download_count  : 	v.Download_count,
				Digg_rate  		:	fmt.Sprintf("%.2f",(Digg/Play)*100) + "%",
				Share_rate   	:	fmt.Sprintf("%.2f",(Share/Play)*100) + "%",
				Comment_rate   	:	fmt.Sprintf("%.2f",(Comment/Play)*100) + "%",
			}
			ps = append(ps,p)
		}
		t,ok := r.Data.Cursor.(float64)
		if ok {
			cursor = int(t)
			goto re
		}

	} */
	return
}

//导出新抖数据
func (w *DouYinSpider) XinDouExcel(file io.Reader) (err error) {
	f, err := excelize.OpenReader(file)
	if err != nil {
		return
	}
	// 根据指定路径保存文件
	if err := f.SaveAs(fmt.Sprintf("抖音Spider-%s-新抖.xlsx", w.Company)); err != nil {
		w.println(err)
	}
	return err
}

//总来量需要的公司uid
type DouYinAllIdResult struct {
	Sec_uid     string `json:"sec_uid"`
	Status_code int    `json:"status_code"`
}

func (w *DouYinSpider) getAllId() error {
re:
	req, err := http.NewRequest("GET", DouYinAllIdUrl, nil)
	if err != nil {
		return err
	}
	req.Header.Set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
	resp, err := w.Client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		w.println("string(b)1", string(b), req)
		return err
	}

	var a DouYinAllIdResult
	err = json.Unmarshal(b, &a)
	if err != nil {
		w.println("(getAllId)JSON解析失败", string(b))

		return err
	}

	if a.Status_code != 0 {
		w.println("(getAllId)登陆失效，重新登陆中。。。", string(b))
	login:
		if err = w.login3(); err == nil {
			w.println("登陆成功")
		} else {
			w.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
			goto login
		}
		goto re
	}
	w.Sec_uid = a.Sec_uid

	return nil
}

//总来量数据
type DouYinAllDataInfo struct {
	Status_code int `json:"status_code"`
	Items       []struct {
		Data_type   int `json:"data_type"`
		Data_values []int
	} `json:"Items"`
}
type DouYinAllData struct {
	Date            string
	VisitUV         string
	NewFans         string
	Fans            string
	VideoVV         string
	DiggCount       string
	CommentCount    string
	ShareCount      string
	AllCount        string
	AllDiggRate     string
	ConductanceRate string
}

func (w *DouYinSpider) collectDouYinAllDataInfo(date string) (zs []DouYinAllData, err error) {
	zs = make([]DouYinAllData, 30)
	datetime := date + " 00:00:00"
	timeLayout := "2006-01-02 15:04:05"
	loc, _ := time.LoadLocation("Local")
	theTime, _ := time.ParseInLocation(timeLayout, datetime, loc)
	timestamp := theTime.Unix() - 2592000
	//计算开始时间
	dateLayout := "2006-01-02"
	startDate := time.Unix(int64(timestamp), 0).Format(dateLayout)
re:
	data := url.Values{}
	data.Add("data_types", "4,3,5,2,9,10,11")
	data.Add("start_time", startDate)
	data.Add("end_time", date)
	data.Add("sec_uids", w.Sec_uid)
	req, err := http.NewRequest("GET", DouYinAllInfoUrl+"?"+data.Encode(), nil)

	if err != nil {
		return nil, err
	}
	req.Header.Set("content-Type", "application/json;charset=UTF-8")

	resp, err := w.Client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	var r DouYinAllDataInfo
	err = json.Unmarshal(b, &r)

	if err != nil {
		w.println("(collectDouYinAllDataInfo)JSON解析失败", string(b))

		return nil, err
	}
	if r.Status_code != 0 {
		w.println("(collectDouYinAllDataInfo)登陆失效，重新登陆中。。。", string(b), err)
	login:
		if err = w.login3(); err == nil {
			w.println("登陆成功")
		} else {
			w.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
			goto login
		}
		goto re
	}

	timestamp2 := timestamp + 86400
	for i := 0; i < 30; i++ {
		dateTime := time.Unix(int64(timestamp2), 0).Format(dateLayout)
		zs[i] = DouYinAllData{
			Date: dateTime,
		}
		timestamp2 = timestamp2 + 86400
	}
	//var VisitUV,NewFans,Fans,VideoVV,DiggCount,CommentCount,ShareCount,AllCount,AllDiggRate,ConductanceRate	string
	for _, v := range r.Items {
		v.Data_values = append(v.Data_values[:1], v.Data_values[2:]...)
		for ii, vv := range v.Data_values {
			if v.Data_type == 4 { //主页访问UV
				zs[ii].VisitUV = fmt.Sprint(vv)
			}
			if v.Data_type == 3 { //新增粉丝
				zs[ii].NewFans = fmt.Sprint(vv)
			}
			if v.Data_type == 5 { //粉丝数
				zs[ii].Fans = fmt.Sprint(vv)
			}
			if v.Data_type == 2 { //视频播放VV
				zs[ii].VideoVV = fmt.Sprint(vv)
			}
			if v.Data_type == 9 { //获得点赞数
				zs[ii].DiggCount = fmt.Sprint(vv)
			}
			if v.Data_type == 10 { //获得评论数
				zs[ii].CommentCount = fmt.Sprint(vv)
			}
			if v.Data_type == 11 { //获得分享数
				zs[ii].ShareCount = fmt.Sprint(vv)
				if zs[ii].VideoVV != "" {
					VideoVv, _ := strconv.ParseFloat(zs[ii].VideoVV, 64)
					if VideoVv == 0 {
						zs[ii].AllDiggRate = "0%"
					} else {
						zs[ii].AllDiggRate = fmt.Sprintf("%.2f", (float64(vv)/VideoVv)*100) + "%"
					}
				}
			}
		}

	}
	return
}

//评论数据
type DouYinCommentInfo struct {
	Status_code       int    `json:"status_code"`
	Status_msg        string `json:"status_msg"`
	Has_more          bool   `json:"has_more"`
	Comment_info_list []struct {
		Comment_id  string `json:"comment_id"`  //评论ID
		Digg_count  string `json:"digg_count"`  //点赞数
		Reply_count string `json:"reply_count"` //回复数
		Text        string `json:"text"`        //评论内容
		User_info   struct {
			Screen_name string `json:"screen_name"` //用户名
		} `json:"user_info"`
	} `json:"comment_info_list"`
}
type DouYinCommentData struct {
	Comment_id  string //评论ID
	Digg_count  string //点赞数
	Reply_count string //回复数
	Text        string //评论内容
	Screen_name string //用户名
	VideoId     string //视频id
}

func (w *DouYinSpider) collectDouYinCommentInfo(date string) (ms []DouYinCommentData, err error) {
re:
	ms = make([]DouYinCommentData, 0)
	data := url.Values{}
	data.Add("aid", "2906")
	data.Add("count", "20")

	for _, s := range w.videoIdArr {
		data.Set("item_id", s.Sec_item_id)
		cursor := 0
	rePage:
		data.Set("cursor", fmt.Sprint(cursor))
		url := DouYinCommentInfoUrl + "?" + data.Encode()
		var q string
		t, err := w.Wd.ExecuteScript(`return window.byted_acrawler.sign({url:"`+url+`"});`, nil)
		if t != nil {
			q = fmt.Sprint(t)
		}
		req, err := http.NewRequest("GET", url+"&_signature="+q, nil)

		if err != nil {
			return nil, err
		}

		req.Header.Set("referer", "https://creator.douyin.com/following/comment")

		resp, err := w.Client.Do(req)
		if err != nil {
			return nil, err
		}

		defer resp.Body.Close()

		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		var r DouYinCommentInfo
		err = json.Unmarshal(b, &r)

		if err != nil {
			w.println("(collectDouYinAllDataInfo)JSON解析失败", string(b))

			return nil, err
		}

		if r.Status_code != 0 || r.Status_msg != "" {
			w.println("(collectDouYinAllDataInfo)登陆失效，重新登陆中。。。", string(b), err)
		login:
			if err = w.login3(); err == nil {
				w.println("登陆成功")
			} else {
				w.println("登陆失败，err= " + err.Error() + "重新登陆。。。")
				goto login
			}
			goto re
		}
		for _, v := range r.Comment_info_list {
			m := DouYinCommentData{
				Comment_id:  v.Comment_id,
				Digg_count:  v.Digg_count,
				Reply_count: v.Reply_count,
				Text:        v.Text,
				Screen_name: v.User_info.Screen_name,
				VideoId:     s.Id,
			}
			ms = append(ms, m)
		}
		if r.Has_more {
			cursor += 20
			goto rePage
		}
	}
	defer w.Wd.Quit()
	return
}

/* 单视频详情
Title				string
Duration			string
Link				string
Date 				string
VideoId 			string
ViewCount 			string
LikeCount			string
CommentCount		string
ShareCount			string
LikeRate			string
CommentRate			string
ShareRate			string
InteractionRate		string
PlayCompletionRate	string
AveragePlayTime		string
VideoAllTime		string
VideoWithFans		string
*/

/* 游戏来量
Mount  		string
Date   		string
AppName   	string
AppId   	string

*/

/*	单视频汇总
Title  				string
Create_time   		string
Is_reviewed   		string
Share_url   		string
Play_count   		string
Digg_count   		string
Comment_count   	string
Forward_count   	string
Share_count   		string
Download_count   	string
Item_id   			string
Digg_rate  			string
Share_rate   		string
Comment_rate   		string
*/

/* 总计
Date  				string
VisitUV   			string
NewFans   			string
Fans   				string
VideoVV   			string
DiggCount   		string
CommentCount   		string
ShareCount   		string
AllCount   			string
AllDiggRate   		string
ConductanceRate		string
*/

func GetLetter(n int) string {
	if n < 26 {
		return string(n + 65)
	}
	t1 := n / 26
	t2 := n % 26
	return GetLetter(t1-1) + GetLetter(t2)
}
func (w *DouYinSpider) exportExcel(cs [][]DouYinView, as [][]DouYinDevData, zs []DouYinAllData) (err error) {
	//func (w *DouYinSpider) exportExcel(cs [][]DouYinView,ps []XinDouData,zs []DouYinAllData) ( err error) {
	//func (w *DouYinSpider) exportExcel(cs [][]DouYinView,zs []DouYinAllData) ( err error) {

	f := excelize.NewFile()
	// 创建一个工作表
	index := f.NewSheet("视频详情")
	f.NewSheet("游戏来量")
	//f.NewSheet("单视频汇总")
	f.NewSheet("总计")
	//设置列宽
	f.SetColWidth("视频详情", "A", "AE", 12)
	f.SetColWidth("游戏来量", "A", "A", 15)
	f.SetColWidth("单视频汇总", "A", "A", 75)
	f.SetColWidth("单视频汇总", "B", "B", 22)
	f.SetColWidth("单视频汇总", "C", "o", 12)
	f.SetColWidth("总计", "A", "A", 15)

	//视频详情
	data := make([][]DouYinView, 0)
	for _, q := range cs {
		data2 := make([]DouYinView, 0)
		for _, qq := range q {
			mount, _ := strconv.Atoi(qq.ViewCount)
			if mount != 0 || len(data2) == 0 {
				data2 = append(data2, qq)
			}
		}
		if len(data2) != 0 {
			data = append(data, data2)
		}

	}
	if len(data) == 0 || len(data[0]) == 0 {
		w.println("今日抖音数据还未更新")
		return err
	}
	for i, v := range data {
		/* //合并单元格
		f.MergeCell("视频详情", fmt.Sprintf("B%d",i*17 + 1), fmt.Sprintf("AE%d",i*17 + 1))
		f.MergeCell("视频详情", fmt.Sprintf("B%d",i*17 + 2), fmt.Sprintf("AE%d",i*17 + 2))
		f.MergeCell("视频详情", fmt.Sprintf("B%d",i*17 + 3), fmt.Sprintf("AE%d",i*17 + 3)) */
		//表头
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+1), "视频标题")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+2), "绑定游戏")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+3), "发布时间")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+4), "日期")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+5), "播放数")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+6), "点赞数")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+7), "评论数")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+8), "分享数")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+9), "点赞率")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+10), "评论率")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+11), "分享率")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+12), "互动率")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+13), "播放完成率")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+14), "平均播放时长")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+15), "视频总时长")
		f.SetCellValue("视频详情", fmt.Sprintf("A%d", i*17+16), "视频带粉")

		f.SetCellValue("视频详情", fmt.Sprintf("C%d", i*17+1), v[0].Title)
		f.SetCellValue("视频详情", fmt.Sprintf("C%d", i*17+2), "")
		f.SetCellValue("视频详情", fmt.Sprintf("C%d", i*17+3), w.videoIdMap[v[0].VideoId].Create_time)

		totalViewCount := 0.00
		totalLikeCount := 0.00
		totalCommentCount := 0.00
		totalShareCount := 0.00
		totalVideoAllTime := 0
		totalVideoWithFans := 0
		var totalLikeRate, totalCommentRate, totalShareRate, totalInteractionRate string
		for ii, vv := range v {
			intView, _ := strconv.ParseFloat(vv.ViewCount, 64)
			intLike, _ := strconv.ParseFloat(vv.LikeCount, 64)
			intComment, _ := strconv.ParseFloat(vv.CommentCount, 64)
			intShare, _ := strconv.ParseFloat(vv.ShareCount, 64)
			intAllTime, _ := strconv.Atoi(vv.VideoAllTime)
			intWithFans, _ := strconv.Atoi(vv.VideoWithFans)
			totalViewCount += intView
			totalLikeCount += intLike
			totalCommentCount += intComment
			totalShareCount += intShare
			totalVideoAllTime += intAllTime
			totalVideoWithFans += intWithFans
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+4), vv.Date)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+5), vv.ViewCount)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+6), vv.LikeCount)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+7), vv.CommentCount)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+8), vv.ShareCount)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+9), vv.LikeRate)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+10), vv.CommentRate)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+11), vv.ShareRate)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+12), vv.InteractionRate)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+13), vv.PlayCompletionRate)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+14), vv.AveragePlayTime)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+15), vv.VideoAllTime)
			f.SetCellValue("视频详情", fmt.Sprintf("%s%d", GetLetter(ii+2), i*17+16), vv.VideoWithFans)
		}

		if totalViewCount > 0 {
			totalLikeRate = fmt.Sprintf("%.2f", (totalLikeCount/totalViewCount)*100) + "%"
			totalCommentRate = fmt.Sprintf("%.2f", (totalCommentCount/totalViewCount)*100) + "%"
			totalShareRate = fmt.Sprintf("%.2f", (totalShareCount/totalViewCount)*100) + "%"
			totalInteractionRate = fmt.Sprintf("%.2f", ((totalLikeCount+totalCommentCount+totalShareCount)/totalViewCount)*100) + "%"
		}

		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+4), "总计")
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+5), fmt.Sprint(int(totalViewCount)))
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+6), fmt.Sprint(int(totalLikeCount)))
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+7), fmt.Sprint(int(totalCommentCount)))
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+8), fmt.Sprint(int(totalShareCount)))
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+9), totalLikeRate)
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+10), totalCommentRate)
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+11), totalShareRate)
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+12), totalInteractionRate)
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+15), fmt.Sprint(totalVideoAllTime))
		f.SetCellValue("视频详情", fmt.Sprintf("B%d", i*17+16), fmt.Sprint(totalVideoWithFans))
	}

	//游戏来量
	f.SetCellValue("游戏来量", "A1", "日期")
	f.MergeCell("游戏来量", "A1", "A2")
	flagDate := false
	for i, m := range as {
		totalNum := 0
		f.MergeCell("游戏来量", fmt.Sprintf("%s1", GetLetter(i*4+1)), fmt.Sprintf("%s1", GetLetter(i*4+4)))

		f.SetCellValue("游戏来量", fmt.Sprintf("%s1", GetLetter(i*4+1)), m[0].AppName)
		f.SetCellValue("游戏来量", fmt.Sprintf("%s2", GetLetter(i*4+1)), "视频来量")
		f.SetCellValue("游戏来量", fmt.Sprintf("%s2", GetLetter(i*4+2)), "转化率")
		f.SetCellValue("游戏来量", fmt.Sprintf("%s2", GetLetter(i*4+3)), "新增视频")
		f.SetCellValue("游戏来量", fmt.Sprintf("%s2", GetLetter(i*4+4)), "备注")
		for ii, mm := range m {
			oneNum, err := strconv.Atoi(mm.Mount)
			if err != nil {
				continue
			}
			totalNum += oneNum
			f.SetCellValue("游戏来量", fmt.Sprintf("%s%d", GetLetter(i*4+1), ii+3), mm.Mount)
			f.SetCellValue("游戏来量", fmt.Sprintf("%s%d", GetLetter(i*4+2), ii+3), "")
			f.SetCellValue("游戏来量", fmt.Sprintf("%s%d", GetLetter(i*4+3), ii+3), "")
			f.SetCellValue("游戏来量", fmt.Sprintf("%s%d", GetLetter(i*4+4), ii+3), "")

		}
		if !flagDate {
			flagDate = true
			for ii, mm := range m {
				f.SetCellValue("游戏来量", fmt.Sprintf("A%d", ii+3), mm.Date)
			}
			f.SetCellValue("游戏来量", fmt.Sprintf("A%d", len(m)+3), "汇总")
		}
		//单个游戏视频来量汇总
		f.SetCellValue("游戏来量", fmt.Sprintf("%s%d", GetLetter(i*4+1), len(m)+3), fmt.Sprint(totalNum))

	}

	//单视频汇总
	/* f.SetCellValue("单视频汇总", "A1", "视频标题")
	f.SetCellValue("单视频汇总", "B1", "发布时间")
	f.SetCellValue("单视频汇总", "C1", "是否已审核")
	f.SetCellValue("单视频汇总", "D1", "视频链接")
	f.SetCellValue("单视频汇总", "E1", "播放数")
	f.SetCellValue("单视频汇总", "F1", "获赞数")
	f.SetCellValue("单视频汇总", "G1", "评论数")
	f.SetCellValue("单视频汇总", "H1", "转发数")
	f.SetCellValue("单视频汇总", "I1", "分享数")
	f.SetCellValue("单视频汇总", "J1", "下载数")
	f.SetCellValue("单视频汇总", "K1", "点赞率")
	f.SetCellValue("单视频汇总", "L1", "分享率")
	f.SetCellValue("单视频汇总", "M1", "评论率")
	f.SetCellValue("单视频汇总", "N1", "播放完成率")
	f.SetCellValue("单视频汇总", "O1", "绑定游戏")

	for i,v := range ps{
		f.SetCellValue("单视频汇总", fmt.Sprintf("A%d",i+2), v.Title)
		f.SetCellValue("单视频汇总", fmt.Sprintf("B%d",i+2), v.Create_time)
		f.SetCellValue("单视频汇总", fmt.Sprintf("C%d",i+2), v.Is_reviewed)
		f.SetCellValue("单视频汇总", fmt.Sprintf("D%d",i+2), v.Share_url)
		f.SetCellValue("单视频汇总", fmt.Sprintf("E%d",i+2), v.Play_count)
		f.SetCellValue("单视频汇总", fmt.Sprintf("F%d",i+2), v.Digg_count)
		f.SetCellValue("单视频汇总", fmt.Sprintf("G%d",i+2), v.Comment_count)
		f.SetCellValue("单视频汇总", fmt.Sprintf("H%d",i+2), v.Forward_count)
		f.SetCellValue("单视频汇总", fmt.Sprintf("I%d",i+2), v.Share_count)
		f.SetCellValue("单视频汇总", fmt.Sprintf("J%d",i+2), v.Download_count)
		f.SetCellValue("单视频汇总", fmt.Sprintf("K%d",i+2), v.Digg_rate)
		f.SetCellValue("单视频汇总", fmt.Sprintf("L%d",i+2), v.Share_rate)
		f.SetCellValue("单视频汇总", fmt.Sprintf("M%d",i+2), v.Comment_rate)
		f.SetCellValue("单视频汇总", fmt.Sprintf("N%d",i+2), "")
		f.SetCellValue("单视频汇总", fmt.Sprintf("O%d",i+2), "")
	} */

	//总计
	f.SetCellValue("总计", "A1", "日期")
	f.SetCellValue("总计", "B1", "主页访问UV")
	f.SetCellValue("总计", "C1", "新增粉丝")
	f.SetCellValue("总计", "D1", "粉丝数")
	f.SetCellValue("总计", "E1", "视频播放VV")
	f.SetCellValue("总计", "F1", "获得点赞数")
	f.SetCellValue("总计", "G1", "获得评论数")
	f.SetCellValue("总计", "H1", "获得分享数")
	f.SetCellValue("总计", "I1", "游戏总来量")
	f.SetCellValue("总计", "J1", "总点赞率")
	f.SetCellValue("总计", "K1", "导量转化率")

	for i, v := range zs {
		f.SetCellValue("总计", fmt.Sprintf("A%d", i+2), v.Date)
		f.SetCellValue("总计", fmt.Sprintf("B%d", i+2), v.VisitUV)
		f.SetCellValue("总计", fmt.Sprintf("C%d", i+2), v.NewFans)
		f.SetCellValue("总计", fmt.Sprintf("D%d", i+2), v.Fans)
		f.SetCellValue("总计", fmt.Sprintf("E%d", i+2), v.VideoVV)
		f.SetCellValue("总计", fmt.Sprintf("F%d", i+2), v.DiggCount)
		f.SetCellValue("总计", fmt.Sprintf("G%d", i+2), v.CommentCount)
		f.SetCellValue("总计", fmt.Sprintf("H%d", i+2), v.ShareCount)
		f.SetCellValue("总计", fmt.Sprintf("I%d", i+2), v.AllCount)
		f.SetCellValue("总计", fmt.Sprintf("J%d", i+2), v.AllDiggRate)
		f.SetCellValue("总计", fmt.Sprintf("K%d", i+2), v.ConductanceRate)
	}

	// 设置工作簿的默认工作表
	f.SetActiveSheet(index)
	// 根据指定路径保存文件
	if err := f.SaveAs(fmt.Sprintf("抖音Spider-%s.xlsx", w.Company)); err != nil {
		w.println(err)
	}
	return err
}
func (w *DouYinSpider) exportCommentExcel(ms []DouYinCommentData) (err error) {
	f := excelize.NewFile()
	// 创建一个工作表
	for k, s := range w.videoIdArr {
		if k == 0 {
			f.SetSheetName("Sheet1", "表"+fmt.Sprint(k+1))
		} else {
			f.NewSheet("表" + fmt.Sprint(k+1))
		}
		f.SetColWidth("表"+fmt.Sprint(k+1), "A", "A", 35)
		f.SetColWidth("表"+fmt.Sprint(k+1), "B", "B", 70)
		f.SetColWidth("表"+fmt.Sprint(k+1), "C", "C", 10)
		f.SetColWidth("表"+fmt.Sprint(k+1), "D", "D", 10)
		f.SetColWidth("表"+fmt.Sprint(k+1), "E", "E", 50)

		f.SetCellValue("表"+fmt.Sprint(k+1), "A1", "用户名")
		f.SetCellValue("表"+fmt.Sprint(k+1), "B1", "评论内容")
		f.SetCellValue("表"+fmt.Sprint(k+1), "C1", "点赞数")
		f.SetCellValue("表"+fmt.Sprint(k+1), "D1", "回复数")
		f.SetCellValue("表"+fmt.Sprint(k+1), "E1", s.Title)
		index := 0
		for _, v := range ms {
			if v.VideoId == s.Id {
				f.SetCellValue("表"+fmt.Sprint(k+1), fmt.Sprintf("A%d", index+2), v.Screen_name)
				f.SetCellValue("表"+fmt.Sprint(k+1), fmt.Sprintf("B%d", index+2), v.Text)
				f.SetCellValue("表"+fmt.Sprint(k+1), fmt.Sprintf("C%d", index+2), v.Digg_count)
				f.SetCellValue("表"+fmt.Sprint(k+1), fmt.Sprintf("D%d", index+2), v.Reply_count)
				index++
			}
		}
	}

	// 设置工作簿的默认工作表
	f.SetActiveSheet(0)
	// 根据指定路径保存文件
	if err := f.SaveAs(fmt.Sprintf("抖音Spider-%s-抖音评论.xlsx", w.Company)); err != nil {
		w.println(err)
	}
	return err
}

func (w *DouYinSpider) collect(date string) (bool, error) {
	/* --------------------开发者平台----------------- */
	//缓存了开发者平台的cookie，如果失效则登录重新缓存
	err := w.getAppInfo()
	if err != nil {
		return false, err
	}
	as, err := w.collectDevDataInfo(date)
	if err != nil {
		return false, err
	}
	/* ---------------------------------------------- */

	/* --------------------创作者平台----------------- */
	//w.println("getVideoId")

	err = w.getVideoId()
	if err != nil {
		return false, err
	}
	ms, err := w.collectDouYinCommentInfo(date)
	if err != nil {
		return false, err
	}
	cs, err := w.collectView(date)
	if err != nil {
		return false, err
	}

	/* ---------------------------------------------- */

	/* --------------------新抖平台------------------- */
	//w.println("getOpenId")

	err = w.getOpenId()
	if err != nil {
		return false, err
	}
	//w.println("collectXinDouDataInfo")

	_, err = w.collectXinDouDataInfo(date)
	if err != nil {
		return false, err
	}
	/* ---------------------------------------------- */

	/* --------------------总来量--------------------- */
	//w.println("getAllId")

	err = w.getAllId()
	if err != nil {
		return false, err
	}
	//w.println("collectDouYinAllDataInfo")

	zs, err := w.collectDouYinAllDataInfo(date)
	if err != nil {
		return false, err
	}

	/* ---------------------------------------------- */

	/* --------------------数据导出------------------- */

	w.println("开始导出表格")

	err = w.exportExcel(cs, as, zs)
	//err = w.exportExcel(cs,nil,nil,nil)
	if err != nil {
		return false, err
	}
	//评论数据表
	err = w.exportCommentExcel(ms)
	if err != nil {
		return false, err
	}
	return true, nil
}

// 执行任务
func (w *DouYinSpider) Run() {
	go func() {
		for {
			<-w.doFlag
			// 采集前一天的数据  因为数据可能被修正
			/* d := time.Now().Add(-48 * time.Hour).Format("2006-01-02")
			w.collect(d) */

			date := time.Now().Add(-24 * time.Hour).Format("2006-01-02")
			w.println("日期:", date, w.Company+"开始采集数据...")
			succ, err := w.collect(date)
			if succ {
				w.SetDone(date)
				w.println("日期:", date, w.Company+"采集数据完成")
			} else {
				if err == nil {
					w.println("日期:", date, "数据未更新 20分钟后重新采集...")
				} else {
					w.println("日期:", date, "出现错误err:", err, "20分钟后重新采集...")
				}
			}
			//time.Sleep(20*time.Minute)
		}
	}()

}
